Libraries

# Data manipulation
library(plyr)
library(dplyr)
library(tidyr)
library(reshape2)
library(tibble)

# Fetching data
library(curl)

# Computing hashes, used for efficient model caching
library(hashr)

# Easier plotting
library(ggplot2); theme_set(theme_minimal())
library(likert)
library(bayesplot)
library(ggpubr)
library(HDInterval)
library(ggridges)

# Baysian modeling
library(brms)

Data fetching

We begin by importing the csv data from the data repository.

d.orig <- read.csv(curl("https://raw.githubusercontent.com/BrokenWindowsInvestigation/Data/master/data.csv"))
d.orig

Encoding the data with correct types

### Utility functions for encoding ###
encode.categorical <- function(column, categories) {
  factor(column, level = categories)
}

encode.bool <- function(column) {
  encode.categorical(column, c("true", "false"))
}
encode.logic <- function(column) {
  encode.categorical(column, c(TRUE, FALSE))
}

encode.orderedcategorical <- function(column, categories) {
  as.ordered(encode.categorical(column, categories))
}

encode.likert <- function(column) {
  encode.orderedcategorical(column, c(-3, -2, -1, 0, 1, 2, 3))
}

### Encode the original data ###
d <- data.frame(
  session = factor(d.orig$session), 
  
  time = d.orig$time,
  
  reused_logic_constructor = encode.bool(d.orig$reused_logic_constructor),
  reused_logic_validation  = encode.bool(d.orig$reused_logic_validation),
  
  equals.state   = encode.orderedcategorical(
    d.orig$equals_state, 
    c("Not implemented", "Duplicated", "Good")
  ),
  hashcode.state = encode.orderedcategorical(
    d.orig$hashcode_state, 
    c("Not implemented", "Duplicated", "Good")
  ),
  
  documentation = factor(d.orig$documentation),
  
  var_names_copied_all        = d.orig$var_names_copied_all,
  var_names_copied_good       = d.orig$var_names_copied_good,
  var_names_copied_good.ratio = d.orig$var_names_copied_good / d.orig$var_names_copied_all,
  var_names_new_all           = d.orig$var_names_new_all,
  var_names_new_good          = d.orig$var_names_new_good,
  var_names_new_good.ratio    = d.orig$var_names_new_good / d.orig$var_names_new_all,
  var_names_edited_all        = d.orig$var_names_edited_all,
  var_names_edited_good       = d.orig$var_names_edited_good,
  var_names_edited_good.ratio = d.orig$var_names_edited_good / d.orig$var_names_edited_all,
  
  sonarqube_issues          = 
    d.orig$sonarqube_issues_major + 
    d.orig$sonarqube_issues_minor + 
    d.orig$sonarqube_issues_info + 
    d.orig$sonarqube_issues_critical,
  sonarqube_issues.major    = d.orig$sonarqube_issues_major,
  sonarqube_issues.minor    = d.orig$sonarqube_issues_minor,
  sonarqube_issues.info     = d.orig$sonarqube_issues_info,
  sonarqube_issues.critical = d.orig$sonarqube_issues_critical,
  
  group = factor(d.orig$group),
  
  education_level = encode.orderedcategorical(d.orig$education_level, c(
    "None", 
    "Some bachelor studies", 
    "Bachelor degree", 
    "Some master studies", 
    "Master degree", 
    "Some Ph.D. studies", 
    "Ph. D."
  )),
  education_field = factor(d.orig$education_field),
  
  work_domain                 = factor(d.orig$work_domain),
  work_experience_programming = d.orig$work_experience_programming,
  work_experience_java        = d.orig$work_experience_java,
  
  workplace_pair_programming = encode.bool(d.orig$workplace_pair_programming),
  workplace_peer_review      = encode.bool(d.orig$workplace_peer_review),
  workplace_td_tracking      = encode.bool(d.orig$workplace_td_tracking),
  workplace_coding_standards = encode.bool(d.orig$workplace_coding_standards),
  
  task_completion = encode.orderedcategorical(d.orig$task_completion, c(
    "Not submitted", 
    "Does not compile", 
    "Invalid solution", 
    "Completed"
  )),
  
  quality_pre_task  = encode.likert(d.orig$quality_pre_task),
  quality_post_task = encode.likert(d.orig$quality_post_task),
  
  high_debt_version = encode.bool(d.orig$high_debt_version),
  scenario          = encode.categorical(d.orig$scenario, c("booking", "tickets")),
  order             = encode.orderedcategorical(d.orig$order, c(0, 1)),
  
  modified_lines         = d.orig$modified_lines,
  large_structure_change = encode.bool(d.orig$large_structure_change)
)

d$equals.exists <- encode.logic(d$equals.state != "Not implemented")
d$hashcode.exists <-encode.logic(d$hashcode.state != "Not implemented")

str(d)
## 'data.frame':    73 obs. of  41 variables:
##  $ session                    : Factor w/ 43 levels "6033c6fc5af2c702367b3a93",..: 1 1 2 2 3 3 4 4 5 6 ...
##  $ time                       : int  1055 2263 3249 12 1382 1197 3855 1984 0 5309 ...
##  $ reused_logic_constructor   : Factor w/ 2 levels "true","false": 1 2 2 2 1 1 1 1 2 2 ...
##  $ reused_logic_validation    : Factor w/ 2 levels "true","false": 1 2 2 2 1 1 1 1 2 1 ...
##  $ equals.state               : Ord.factor w/ 3 levels "Not implemented"<..: 2 2 1 1 1 3 1 3 1 3 ...
##  $ hashcode.state             : Ord.factor w/ 3 levels "Not implemented"<..: 2 2 1 1 1 3 1 3 1 3 ...
##  $ documentation              : Factor w/ 4 levels "Correct","Incorrect",..: 3 2 2 3 1 3 3 2 4 3 ...
##  $ var_names_copied_all       : int  6 12 15 0 8 7 5 7 0 9 ...
##  $ var_names_copied_good      : int  6 1 3 0 8 5 3 7 0 1 ...
##  $ var_names_copied_good.ratio: num  1 0.0833 0.2 NaN 1 ...
##  $ var_names_new_all          : int  4 3 4 0 4 2 4 2 0 22 ...
##  $ var_names_new_good         : int  4 1 0 0 4 2 4 2 0 11 ...
##  $ var_names_new_good.ratio   : num  1 0.333 0 NaN 1 ...
##  $ var_names_edited_all       : int  0 0 0 0 0 0 0 0 0 0 ...
##  $ var_names_edited_good      : int  0 0 0 0 0 0 0 0 0 0 ...
##  $ var_names_edited_good.ratio: num  NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN ...
##  $ sonarqube_issues           : int  0 4 5 0 0 3 1 0 0 12 ...
##  $ sonarqube_issues.major     : int  0 2 5 0 0 3 1 0 0 8 ...
##  $ sonarqube_issues.minor     : int  0 2 0 0 0 0 0 0 0 4 ...
##  $ sonarqube_issues.info      : int  0 0 0 0 0 0 0 0 0 0 ...
##  $ sonarqube_issues.critical  : int  0 0 0 0 0 0 0 0 0 0 ...
##  $ group                      : Factor w/ 7 levels "code-interested",..: 7 7 7 7 7 7 7 7 7 7 ...
##  $ education_level            : Ord.factor w/ 6 levels "None"<"Some bachelor studies"<..: 3 3 4 4 4 4 2 2 2 4 ...
##  $ education_field            : Factor w/ 7 levels "Civil Engineering",..: 2 2 2 2 2 2 7 7 7 7 ...
##  $ work_domain                : Factor w/ 16 levels "Adtech","App",..: 12 12 3 3 7 7 3 3 12 16 ...
##  $ work_experience_programming: num  0 0 0.2 0.2 1 1 0 0 0 2 ...
##  $ work_experience_java       : num  0 0 0 0 0 0 0 0 0 0 ...
##  $ workplace_pair_programming : Factor w/ 2 levels "true","false": 1 1 2 2 2 2 2 2 2 2 ...
##  $ workplace_peer_review      : Factor w/ 2 levels "true","false": 2 2 1 1 2 2 2 2 2 2 ...
##  $ workplace_td_tracking      : Factor w/ 2 levels "true","false": 2 2 2 2 2 2 2 2 2 2 ...
##  $ workplace_coding_standards : Factor w/ 2 levels "true","false": 2 2 1 1 2 2 1 1 2 2 ...
##  $ task_completion            : Ord.factor w/ 4 levels "Not submitted"<..: 2 4 3 1 4 4 4 4 1 4 ...
##  $ quality_pre_task           : Ord.factor w/ 7 levels "-3"<"-2"<"-1"<..: 4 3 6 6 4 2 2 6 4 3 ...
##  $ quality_post_task          : Ord.factor w/ 7 levels "-3"<"-2"<"-1"<..: 1 1 3 4 4 4 4 4 4 6 ...
##  $ high_debt_version          : Factor w/ 2 levels "true","false": 2 1 1 2 2 1 1 2 2 1 ...
##  $ scenario                   : Factor w/ 2 levels "booking","tickets": 1 2 1 2 1 2 1 2 1 1 ...
##  $ order                      : Ord.factor w/ 2 levels "0"<"1": 2 1 1 2 1 2 2 1 1 2 ...
##  $ modified_lines             : int  75 71 78 0 41 22 18 32 0 246 ...
##  $ large_structure_change     : Factor w/ 2 levels "true","false": 2 2 2 2 2 2 2 2 2 1 ...
##  $ equals.exists              : Factor w/ 2 levels "TRUE","FALSE": 1 1 2 2 2 1 2 1 2 1 ...
##  $ hashcode.exists            : Factor w/ 2 levels "TRUE","FALSE": 1 1 2 2 2 1 2 1 2 1 ...

Partial data sets and aggregates

For some models partial data sets and aggregates are needed.

Sessions as rows

d.sessions <- d %>% group_by(session) %>% dplyr::summarise(
  across(task_completion, min),
  across(c(
    education_level, 
    education_field, 
    work_domain, 
    group,
    work_experience_java, 
    work_experience_programming, 
    workplace_coding_standards, 
    workplace_pair_programming, 
    workplace_peer_review, 
    workplace_td_tracking
  ), first)
  ) 

d$work_experience_programming.s = scale(d$work_experience_programming)
d$work_experience_java.s = scale(d$work_experience_java)

d.sessions

Sessions as rows (only completed)

d.sessions.completed <- d.sessions %>% filter(task_completion == "Completed")

d.sessions.completed

Only completed

d.completed <- d %>% filter(task_completion == "Completed")

d.completed$work_experience_programming.s = scale(d.completed$work_experience_programming)
d.completed$work_experience_java.s = scale(d.completed$work_experience_java)
d.completed$time.s = scale(d.completed$time)
d.completed$sonarqube_issues.s = scale(d.completed$sonarqube_issues)

d.completed

Only both completed

d.both_completed <- d %>% semi_join(d.sessions.completed, by = "session")

d.both_completed$work_experience_programming.s = scale(d.both_completed$work_experience_programming)
d.both_completed$work_experience_java.s = scale(d.both_completed$work_experience_java)
d.both_completed$time.s = scale(d.both_completed$time)
d.both_completed$sonarqube_issues.s = scale(d.both_completed$sonarqube_issues)

d.both_completed

Model expansion utility function

The function extendable_model takes some basic arguments for creating brms models and returns a function that can be called with additioanl parameters to combine with those passed to extendable_model. The extendable_model takes the following arguments:

  • base_name is a name that is used to identify this extendable model while caching.
  • base_formula the formula that will be extended and passed to brms::brm, represented as string.
  • data the data frame to be passed to brms::brm.
  • base_priors (NULL) is a vector of priors to be passed to brms::brm.
  • base_control (NULL) is a vector of control options to be passed to brms::brm.

The returned function takes the following arguments:

  • additional_variables (NULL) a vector of aditional variables (predictors) to pass to pass to brms::brm in adition to base_formula.
  • additional_priors (NULL) a vector of additioanl priors to pass to brms::brm in adition to base_priors.
  • only_priors (FALSE) indicates if the model should be epty and not compiled, usefull to extract default priors of a model.
  • sample_prior ("no") is passed to the sample_prior of brms::brm.
  • control_override (NULL) takes a vector of control arguments for brms::brm that will override base_control.
extendable_model <- function(
  base_name, 
  base_formula, 
  family, 
  data, 
  base_priors = NULL, 
  base_control = NULL
) {
  function(
    additional_variables = NULL, 
    additional_priors = NULL, 
    only_priors = FALSE, 
    sample_prior = "no", 
    control_override = NULL
  ) {
    # Sort variable names for consistent caching and naming
    additional_variables.sorted <- sort(additional_variables)
    
    # Build priors
    priors <- base_priors
    if (!is.null(additional_priors)) {
      priors <- c(base_priors, additional_priors)
    }
    if (only_priors) {
      priors <- NULL
    }
    
    # Build formula
    additional_variables.formula <- paste(additional_variables.sorted, collapse = " + ")
    formula <- base_formula
    if (!is.null(additional_variables)) {
      formula <- paste(base_formula, additional_variables.formula, sep = " + ")
    }
    
    # Build cache file name
    additional_variables.name <- paste(additional_variables.sorted, collapse = ".")
    name <- base_name
    if (!is.null(additional_variables)) {
      name <- paste(base_name, hash(additional_variables.name), sep = ".")
    }
    name <- paste(name, paste("sample_priors-", sample_prior, sep = ""), sep = ".")
    name <- paste(name, paste("priors_hash-", hash(priors), sep = ""), sep = ".")
    name <- paste(name, paste("formula_hash-", hash(formula), sep = ""), sep = ".")
    
    # Get control options
    control <- base_control
    if (!is.null(control_override)) {
      control <- control_override
    }
    
    # Create and return the brms model
    brm(
      formula = as.formula(formula),
      family = family,
      data = as.data.frame(data),
      prior = priors,
      empty = only_priors,
      sample_prior = sample_prior,
      file = paste("fits", name, sep = "/"),
      file_refit = "on_change",
      seed = 20210421,
      control = control
    )
  }
}

Example usage:

## Not run
m.with <- extendable_model(
  base_name = "m", 
  base_formula = "time ~ 1",
  family = negbinomial(), 
  data = d.both_completed, 
  base_priors = c(
    prior(normal(0, 1), class = "Intercept")
  )
)

prior_summary(m.with(only_priors = TRUE))

pp_check(m.with(sample_prior = "only"), nsamples = 200)

summary(m.with())

pp_check(m.with(), nsamples = 200)

loo(
  m.with(),
  m.with("high_debt_version"),
  m.with(c("high_debt_version", "scenario"))
)
## End(Not run)
LS0tCnRpdGxlOiAiU2V0dXAgJiBEYXRhIFByZXBhcmF0aW9uIgphdXRob3I6IEhhbXB1cyBCcm9tYW4gJiBXaWxsaWFtIExldsOpbgpkYXRlOiAyMDIxLTA1Cm91dHB1dDogCiAgaHRtbF9kb2N1bWVudDogCiAgICBwYW5kb2NfYXJnczogWyAiLW8iLCAiZG9jcy9zZXR1cC5odG1sIiBdCi0tLQojIyBMaWJyYXJpZXMKCmBgYHtyIGxvYWQtbGlicmFyaWVzLCBjbGFzcy5zb3VyY2UgPSAnZm9sZC1zaG93JywgbWVzc2FnZT1GQUxTRX0KIyBEYXRhIG1hbmlwdWxhdGlvbgpsaWJyYXJ5KHBseXIpCmxpYnJhcnkoZHBseXIpCmxpYnJhcnkodGlkeXIpCmxpYnJhcnkocmVzaGFwZTIpCmxpYnJhcnkodGliYmxlKQoKIyBGZXRjaGluZyBkYXRhCmxpYnJhcnkoY3VybCkKCiMgQ29tcHV0aW5nIGhhc2hlcywgdXNlZCBmb3IgZWZmaWNpZW50IG1vZGVsIGNhY2hpbmcKbGlicmFyeShoYXNocikKCiMgRWFzaWVyIHBsb3R0aW5nCmxpYnJhcnkoZ2dwbG90Mik7IHRoZW1lX3NldCh0aGVtZV9taW5pbWFsKCkpCmxpYnJhcnkobGlrZXJ0KQpsaWJyYXJ5KGJheWVzcGxvdCkKbGlicmFyeShnZ3B1YnIpCmxpYnJhcnkoSERJbnRlcnZhbCkKbGlicmFyeShnZ3JpZGdlcykKCiMgQmF5c2lhbiBtb2RlbGluZwpsaWJyYXJ5KGJybXMpCmBgYAoKIyMgRGF0YSBmZXRjaGluZwoKV2UgYmVnaW4gYnkgaW1wb3J0aW5nIHRoZSBjc3YgZGF0YSBmcm9tIHRoZSBbZGF0YSByZXBvc2l0b3J5XShodHRwczovL2dpdGh1Yi5jb20vQnJva2VuV2luZG93c0ludmVzdGlnYXRpb24vRGF0YSkuCgpgYGB7ciBmZXRjaC1kYXRhfQpkLm9yaWcgPC0gcmVhZC5jc3YoY3VybCgiaHR0cHM6Ly9yYXcuZ2l0aHVidXNlcmNvbnRlbnQuY29tL0Jyb2tlbldpbmRvd3NJbnZlc3RpZ2F0aW9uL0RhdGEvbWFzdGVyL2RhdGEuY3N2IikpCmQub3JpZwpgYGAKCiMjIEVuY29kaW5nIHRoZSBkYXRhIHdpdGggY29ycmVjdCB0eXBlcwoKYGBge3IgZGF0YS1lbmNvZGluZ30KIyMjIFV0aWxpdHkgZnVuY3Rpb25zIGZvciBlbmNvZGluZyAjIyMKZW5jb2RlLmNhdGVnb3JpY2FsIDwtIGZ1bmN0aW9uKGNvbHVtbiwgY2F0ZWdvcmllcykgewogIGZhY3Rvcihjb2x1bW4sIGxldmVsID0gY2F0ZWdvcmllcykKfQoKZW5jb2RlLmJvb2wgPC0gZnVuY3Rpb24oY29sdW1uKSB7CiAgZW5jb2RlLmNhdGVnb3JpY2FsKGNvbHVtbiwgYygidHJ1ZSIsICJmYWxzZSIpKQp9CmVuY29kZS5sb2dpYyA8LSBmdW5jdGlvbihjb2x1bW4pIHsKICBlbmNvZGUuY2F0ZWdvcmljYWwoY29sdW1uLCBjKFRSVUUsIEZBTFNFKSkKfQoKZW5jb2RlLm9yZGVyZWRjYXRlZ29yaWNhbCA8LSBmdW5jdGlvbihjb2x1bW4sIGNhdGVnb3JpZXMpIHsKICBhcy5vcmRlcmVkKGVuY29kZS5jYXRlZ29yaWNhbChjb2x1bW4sIGNhdGVnb3JpZXMpKQp9CgplbmNvZGUubGlrZXJ0IDwtIGZ1bmN0aW9uKGNvbHVtbikgewogIGVuY29kZS5vcmRlcmVkY2F0ZWdvcmljYWwoY29sdW1uLCBjKC0zLCAtMiwgLTEsIDAsIDEsIDIsIDMpKQp9CgojIyMgRW5jb2RlIHRoZSBvcmlnaW5hbCBkYXRhICMjIwpkIDwtIGRhdGEuZnJhbWUoCiAgc2Vzc2lvbiA9IGZhY3RvcihkLm9yaWckc2Vzc2lvbiksIAogIAogIHRpbWUgPSBkLm9yaWckdGltZSwKICAKICByZXVzZWRfbG9naWNfY29uc3RydWN0b3IgPSBlbmNvZGUuYm9vbChkLm9yaWckcmV1c2VkX2xvZ2ljX2NvbnN0cnVjdG9yKSwKICByZXVzZWRfbG9naWNfdmFsaWRhdGlvbiAgPSBlbmNvZGUuYm9vbChkLm9yaWckcmV1c2VkX2xvZ2ljX3ZhbGlkYXRpb24pLAogIAogIGVxdWFscy5zdGF0ZSAgID0gZW5jb2RlLm9yZGVyZWRjYXRlZ29yaWNhbCgKICAgIGQub3JpZyRlcXVhbHNfc3RhdGUsIAogICAgYygiTm90IGltcGxlbWVudGVkIiwgIkR1cGxpY2F0ZWQiLCAiR29vZCIpCiAgKSwKICBoYXNoY29kZS5zdGF0ZSA9IGVuY29kZS5vcmRlcmVkY2F0ZWdvcmljYWwoCiAgICBkLm9yaWckaGFzaGNvZGVfc3RhdGUsIAogICAgYygiTm90IGltcGxlbWVudGVkIiwgIkR1cGxpY2F0ZWQiLCAiR29vZCIpCiAgKSwKICAKICBkb2N1bWVudGF0aW9uID0gZmFjdG9yKGQub3JpZyRkb2N1bWVudGF0aW9uKSwKICAKICB2YXJfbmFtZXNfY29waWVkX2FsbCAgICAgICAgPSBkLm9yaWckdmFyX25hbWVzX2NvcGllZF9hbGwsCiAgdmFyX25hbWVzX2NvcGllZF9nb29kICAgICAgID0gZC5vcmlnJHZhcl9uYW1lc19jb3BpZWRfZ29vZCwKICB2YXJfbmFtZXNfY29waWVkX2dvb2QucmF0aW8gPSBkLm9yaWckdmFyX25hbWVzX2NvcGllZF9nb29kIC8gZC5vcmlnJHZhcl9uYW1lc19jb3BpZWRfYWxsLAogIHZhcl9uYW1lc19uZXdfYWxsICAgICAgICAgICA9IGQub3JpZyR2YXJfbmFtZXNfbmV3X2FsbCwKICB2YXJfbmFtZXNfbmV3X2dvb2QgICAgICAgICAgPSBkLm9yaWckdmFyX25hbWVzX25ld19nb29kLAogIHZhcl9uYW1lc19uZXdfZ29vZC5yYXRpbyAgICA9IGQub3JpZyR2YXJfbmFtZXNfbmV3X2dvb2QgLyBkLm9yaWckdmFyX25hbWVzX25ld19hbGwsCiAgdmFyX25hbWVzX2VkaXRlZF9hbGwgICAgICAgID0gZC5vcmlnJHZhcl9uYW1lc19lZGl0ZWRfYWxsLAogIHZhcl9uYW1lc19lZGl0ZWRfZ29vZCAgICAgICA9IGQub3JpZyR2YXJfbmFtZXNfZWRpdGVkX2dvb2QsCiAgdmFyX25hbWVzX2VkaXRlZF9nb29kLnJhdGlvID0gZC5vcmlnJHZhcl9uYW1lc19lZGl0ZWRfZ29vZCAvIGQub3JpZyR2YXJfbmFtZXNfZWRpdGVkX2FsbCwKICAKICBzb25hcnF1YmVfaXNzdWVzICAgICAgICAgID0gCiAgICBkLm9yaWckc29uYXJxdWJlX2lzc3Vlc19tYWpvciArIAogICAgZC5vcmlnJHNvbmFycXViZV9pc3N1ZXNfbWlub3IgKyAKICAgIGQub3JpZyRzb25hcnF1YmVfaXNzdWVzX2luZm8gKyAKICAgIGQub3JpZyRzb25hcnF1YmVfaXNzdWVzX2NyaXRpY2FsLAogIHNvbmFycXViZV9pc3N1ZXMubWFqb3IgICAgPSBkLm9yaWckc29uYXJxdWJlX2lzc3Vlc19tYWpvciwKICBzb25hcnF1YmVfaXNzdWVzLm1pbm9yICAgID0gZC5vcmlnJHNvbmFycXViZV9pc3N1ZXNfbWlub3IsCiAgc29uYXJxdWJlX2lzc3Vlcy5pbmZvICAgICA9IGQub3JpZyRzb25hcnF1YmVfaXNzdWVzX2luZm8sCiAgc29uYXJxdWJlX2lzc3Vlcy5jcml0aWNhbCA9IGQub3JpZyRzb25hcnF1YmVfaXNzdWVzX2NyaXRpY2FsLAogIAogIGdyb3VwID0gZmFjdG9yKGQub3JpZyRncm91cCksCiAgCiAgZWR1Y2F0aW9uX2xldmVsID0gZW5jb2RlLm9yZGVyZWRjYXRlZ29yaWNhbChkLm9yaWckZWR1Y2F0aW9uX2xldmVsLCBjKAogICAgIk5vbmUiLCAKICAgICJTb21lIGJhY2hlbG9yIHN0dWRpZXMiLCAKICAgICJCYWNoZWxvciBkZWdyZWUiLCAKICAgICJTb21lIG1hc3RlciBzdHVkaWVzIiwgCiAgICAiTWFzdGVyIGRlZ3JlZSIsIAogICAgIlNvbWUgUGguRC4gc3R1ZGllcyIsIAogICAgIlBoLiBELiIKICApKSwKICBlZHVjYXRpb25fZmllbGQgPSBmYWN0b3IoZC5vcmlnJGVkdWNhdGlvbl9maWVsZCksCiAgCiAgd29ya19kb21haW4gICAgICAgICAgICAgICAgID0gZmFjdG9yKGQub3JpZyR3b3JrX2RvbWFpbiksCiAgd29ya19leHBlcmllbmNlX3Byb2dyYW1taW5nID0gZC5vcmlnJHdvcmtfZXhwZXJpZW5jZV9wcm9ncmFtbWluZywKICB3b3JrX2V4cGVyaWVuY2VfamF2YSAgICAgICAgPSBkLm9yaWckd29ya19leHBlcmllbmNlX2phdmEsCiAgCiAgd29ya3BsYWNlX3BhaXJfcHJvZ3JhbW1pbmcgPSBlbmNvZGUuYm9vbChkLm9yaWckd29ya3BsYWNlX3BhaXJfcHJvZ3JhbW1pbmcpLAogIHdvcmtwbGFjZV9wZWVyX3JldmlldyAgICAgID0gZW5jb2RlLmJvb2woZC5vcmlnJHdvcmtwbGFjZV9wZWVyX3JldmlldyksCiAgd29ya3BsYWNlX3RkX3RyYWNraW5nICAgICAgPSBlbmNvZGUuYm9vbChkLm9yaWckd29ya3BsYWNlX3RkX3RyYWNraW5nKSwKICB3b3JrcGxhY2VfY29kaW5nX3N0YW5kYXJkcyA9IGVuY29kZS5ib29sKGQub3JpZyR3b3JrcGxhY2VfY29kaW5nX3N0YW5kYXJkcyksCiAgCiAgdGFza19jb21wbGV0aW9uID0gZW5jb2RlLm9yZGVyZWRjYXRlZ29yaWNhbChkLm9yaWckdGFza19jb21wbGV0aW9uLCBjKAogICAgIk5vdCBzdWJtaXR0ZWQiLCAKICAgICJEb2VzIG5vdCBjb21waWxlIiwgCiAgICAiSW52YWxpZCBzb2x1dGlvbiIsIAogICAgIkNvbXBsZXRlZCIKICApKSwKICAKICBxdWFsaXR5X3ByZV90YXNrICA9IGVuY29kZS5saWtlcnQoZC5vcmlnJHF1YWxpdHlfcHJlX3Rhc2spLAogIHF1YWxpdHlfcG9zdF90YXNrID0gZW5jb2RlLmxpa2VydChkLm9yaWckcXVhbGl0eV9wb3N0X3Rhc2spLAogIAogIGhpZ2hfZGVidF92ZXJzaW9uID0gZW5jb2RlLmJvb2woZC5vcmlnJGhpZ2hfZGVidF92ZXJzaW9uKSwKICBzY2VuYXJpbyAgICAgICAgICA9IGVuY29kZS5jYXRlZ29yaWNhbChkLm9yaWckc2NlbmFyaW8sIGMoImJvb2tpbmciLCAidGlja2V0cyIpKSwKICBvcmRlciAgICAgICAgICAgICA9IGVuY29kZS5vcmRlcmVkY2F0ZWdvcmljYWwoZC5vcmlnJG9yZGVyLCBjKDAsIDEpKSwKICAKICBtb2RpZmllZF9saW5lcyAgICAgICAgID0gZC5vcmlnJG1vZGlmaWVkX2xpbmVzLAogIGxhcmdlX3N0cnVjdHVyZV9jaGFuZ2UgPSBlbmNvZGUuYm9vbChkLm9yaWckbGFyZ2Vfc3RydWN0dXJlX2NoYW5nZSkKKQoKZCRlcXVhbHMuZXhpc3RzIDwtIGVuY29kZS5sb2dpYyhkJGVxdWFscy5zdGF0ZSAhPSAiTm90IGltcGxlbWVudGVkIikKZCRoYXNoY29kZS5leGlzdHMgPC1lbmNvZGUubG9naWMoZCRoYXNoY29kZS5zdGF0ZSAhPSAiTm90IGltcGxlbWVudGVkIikKCnN0cihkKQpgYGAKCiMjIFBhcnRpYWwgZGF0YSBzZXRzIGFuZCBhZ2dyZWdhdGVzIHsudGFic2V0fQoKRm9yIHNvbWUgbW9kZWxzIHBhcnRpYWwgZGF0YSBzZXRzIGFuZCBhZ2dyZWdhdGVzIGFyZSBuZWVkZWQuCgojIyMgU2Vzc2lvbnMgYXMgcm93cwoKYGBge3IgcGFydGlhbC1zZXNzaW9uc30KZC5zZXNzaW9ucyA8LSBkICU+JSBncm91cF9ieShzZXNzaW9uKSAlPiUgZHBseXI6OnN1bW1hcmlzZSgKICBhY3Jvc3ModGFza19jb21wbGV0aW9uLCBtaW4pLAogIGFjcm9zcyhjKAogICAgZWR1Y2F0aW9uX2xldmVsLCAKICAgIGVkdWNhdGlvbl9maWVsZCwgCiAgICB3b3JrX2RvbWFpbiwgCiAgICBncm91cCwKICAgIHdvcmtfZXhwZXJpZW5jZV9qYXZhLCAKICAgIHdvcmtfZXhwZXJpZW5jZV9wcm9ncmFtbWluZywgCiAgICB3b3JrcGxhY2VfY29kaW5nX3N0YW5kYXJkcywgCiAgICB3b3JrcGxhY2VfcGFpcl9wcm9ncmFtbWluZywgCiAgICB3b3JrcGxhY2VfcGVlcl9yZXZpZXcsIAogICAgd29ya3BsYWNlX3RkX3RyYWNraW5nCiAgKSwgZmlyc3QpCiAgKSAKCmQkd29ya19leHBlcmllbmNlX3Byb2dyYW1taW5nLnMgPSBzY2FsZShkJHdvcmtfZXhwZXJpZW5jZV9wcm9ncmFtbWluZykKZCR3b3JrX2V4cGVyaWVuY2VfamF2YS5zID0gc2NhbGUoZCR3b3JrX2V4cGVyaWVuY2VfamF2YSkKCmQuc2Vzc2lvbnMKYGBgCgojIyMgU2Vzc2lvbnMgYXMgcm93cyAob25seSBjb21wbGV0ZWQpCgpgYGB7ciBwYXJ0aWFsLXNlc3Npb25zLWNvbXBsZXRlZH0KZC5zZXNzaW9ucy5jb21wbGV0ZWQgPC0gZC5zZXNzaW9ucyAlPiUgZmlsdGVyKHRhc2tfY29tcGxldGlvbiA9PSAiQ29tcGxldGVkIikKCmQuc2Vzc2lvbnMuY29tcGxldGVkCmBgYAoKIyMjIE9ubHkgY29tcGxldGVkCgpgYGB7ciBwYXJ0aWFsLXN1Ym1pdHRlZH0KZC5jb21wbGV0ZWQgPC0gZCAlPiUgZmlsdGVyKHRhc2tfY29tcGxldGlvbiA9PSAiQ29tcGxldGVkIikKCmQuY29tcGxldGVkJHdvcmtfZXhwZXJpZW5jZV9wcm9ncmFtbWluZy5zID0gc2NhbGUoZC5jb21wbGV0ZWQkd29ya19leHBlcmllbmNlX3Byb2dyYW1taW5nKQpkLmNvbXBsZXRlZCR3b3JrX2V4cGVyaWVuY2VfamF2YS5zID0gc2NhbGUoZC5jb21wbGV0ZWQkd29ya19leHBlcmllbmNlX2phdmEpCmQuY29tcGxldGVkJHRpbWUucyA9IHNjYWxlKGQuY29tcGxldGVkJHRpbWUpCmQuY29tcGxldGVkJHNvbmFycXViZV9pc3N1ZXMucyA9IHNjYWxlKGQuY29tcGxldGVkJHNvbmFycXViZV9pc3N1ZXMpCgpkLmNvbXBsZXRlZApgYGAKCiMjIyBPbmx5IGJvdGggY29tcGxldGVkCmBgYHtyIHBhcnRpYWwtc3VibWl0dGVkLWJvdGh9CmQuYm90aF9jb21wbGV0ZWQgPC0gZCAlPiUgc2VtaV9qb2luKGQuc2Vzc2lvbnMuY29tcGxldGVkLCBieSA9ICJzZXNzaW9uIikKCmQuYm90aF9jb21wbGV0ZWQkd29ya19leHBlcmllbmNlX3Byb2dyYW1taW5nLnMgPSBzY2FsZShkLmJvdGhfY29tcGxldGVkJHdvcmtfZXhwZXJpZW5jZV9wcm9ncmFtbWluZykKZC5ib3RoX2NvbXBsZXRlZCR3b3JrX2V4cGVyaWVuY2VfamF2YS5zID0gc2NhbGUoZC5ib3RoX2NvbXBsZXRlZCR3b3JrX2V4cGVyaWVuY2VfamF2YSkKZC5ib3RoX2NvbXBsZXRlZCR0aW1lLnMgPSBzY2FsZShkLmJvdGhfY29tcGxldGVkJHRpbWUpCmQuYm90aF9jb21wbGV0ZWQkc29uYXJxdWJlX2lzc3Vlcy5zID0gc2NhbGUoZC5ib3RoX2NvbXBsZXRlZCRzb25hcnF1YmVfaXNzdWVzKQoKZC5ib3RoX2NvbXBsZXRlZApgYGAKCiMjIE1vZGVsIGV4cGFuc2lvbiB1dGlsaXR5IGZ1bmN0aW9uCgpUaGUgZnVuY3Rpb24gYGV4dGVuZGFibGVfbW9kZWxgIHRha2VzIHNvbWUgYmFzaWMgYXJndW1lbnRzIGZvciBjcmVhdGluZyBicm1zIG1vZGVscyBhbmQgcmV0dXJucyBhIGZ1bmN0aW9uIHRoYXQgY2FuIGJlIGNhbGxlZCB3aXRoIGFkZGl0aW9hbmwgcGFyYW1ldGVycyB0byBjb21iaW5lIHdpdGggdGhvc2UgcGFzc2VkIHRvIGBleHRlbmRhYmxlX21vZGVsYC4gVGhlIGBleHRlbmRhYmxlX21vZGVsYCB0YWtlcyB0aGUgZm9sbG93aW5nIGFyZ3VtZW50czoKCiogYGJhc2VfbmFtZWAgaXMgYSBuYW1lIHRoYXQgaXMgdXNlZCB0byBpZGVudGlmeSB0aGlzIGV4dGVuZGFibGUgbW9kZWwgd2hpbGUgY2FjaGluZy4KKiBgYmFzZV9mb3JtdWxhYCB0aGUgZm9ybXVsYSB0aGF0IHdpbGwgYmUgZXh0ZW5kZWQgYW5kIHBhc3NlZCB0byBgYnJtczo6YnJtYCwgcmVwcmVzZW50ZWQgYXMgc3RyaW5nLgoqIGBkYXRhYCB0aGUgZGF0YSBmcmFtZSB0byBiZSBwYXNzZWQgdG8gYGJybXM6OmJybWAuCiogYGJhc2VfcHJpb3JzYCAoYE5VTExgKSBpcyBhIHZlY3RvciBvZiBwcmlvcnMgdG8gYmUgcGFzc2VkIHRvIGBicm1zOjpicm1gLgoqIGBiYXNlX2NvbnRyb2xgIChgTlVMTGApIGlzIGEgdmVjdG9yIG9mIGNvbnRyb2wgb3B0aW9ucyB0byBiZSBwYXNzZWQgdG8gYGJybXM6OmJybWAuCgpUaGUgcmV0dXJuZWQgZnVuY3Rpb24gdGFrZXMgdGhlIGZvbGxvd2luZyBhcmd1bWVudHM6CgoqIGBhZGRpdGlvbmFsX3ZhcmlhYmxlc2AgKGBOVUxMYCkgYSB2ZWN0b3Igb2YgYWRpdGlvbmFsIHZhcmlhYmxlcyAocHJlZGljdG9ycykgdG8gcGFzcyB0byBwYXNzIHRvIGBicm1zOjpicm1gIGluIGFkaXRpb24gdG8gYGJhc2VfZm9ybXVsYWAuCiogYGFkZGl0aW9uYWxfcHJpb3JzYCAoYE5VTExgKSBhIHZlY3RvciBvZiBhZGRpdGlvYW5sIHByaW9ycyB0byBwYXNzIHRvIGBicm1zOjpicm1gIGluIGFkaXRpb24gdG8gYGJhc2VfcHJpb3JzYC4KKiBgb25seV9wcmlvcnNgIChgRkFMU0VgKSBpbmRpY2F0ZXMgaWYgdGhlIG1vZGVsIHNob3VsZCBiZSBlcHR5IGFuZCBub3QgY29tcGlsZWQsIHVzZWZ1bGwgdG8gZXh0cmFjdCBkZWZhdWx0IHByaW9ycyBvZiBhIG1vZGVsLgoqIGBzYW1wbGVfcHJpb3JgIChgIm5vImApIGlzIHBhc3NlZCB0byB0aGUgYHNhbXBsZV9wcmlvcmAgb2YgYGJybXM6OmJybWAuCiogYGNvbnRyb2xfb3ZlcnJpZGVgIChgTlVMTGApIHRha2VzIGEgdmVjdG9yIG9mIGBjb250cm9sYCBhcmd1bWVudHMgZm9yIGBicm1zOjpicm1gIHRoYXQgd2lsbCBvdmVycmlkZSBgYmFzZV9jb250cm9sYC4KCmBgYHtyIGV4dGVuZGFibGUtbW9kZWx9CmV4dGVuZGFibGVfbW9kZWwgPC0gZnVuY3Rpb24oCiAgYmFzZV9uYW1lLCAKICBiYXNlX2Zvcm11bGEsIAogIGZhbWlseSwgCiAgZGF0YSwgCiAgYmFzZV9wcmlvcnMgPSBOVUxMLCAKICBiYXNlX2NvbnRyb2wgPSBOVUxMCikgewogIGZ1bmN0aW9uKAogICAgYWRkaXRpb25hbF92YXJpYWJsZXMgPSBOVUxMLCAKICAgIGFkZGl0aW9uYWxfcHJpb3JzID0gTlVMTCwgCiAgICBvbmx5X3ByaW9ycyA9IEZBTFNFLCAKICAgIHNhbXBsZV9wcmlvciA9ICJubyIsIAogICAgY29udHJvbF9vdmVycmlkZSA9IE5VTEwKICApIHsKICAgICMgU29ydCB2YXJpYWJsZSBuYW1lcyBmb3IgY29uc2lzdGVudCBjYWNoaW5nIGFuZCBuYW1pbmcKICAgIGFkZGl0aW9uYWxfdmFyaWFibGVzLnNvcnRlZCA8LSBzb3J0KGFkZGl0aW9uYWxfdmFyaWFibGVzKQogICAgCiAgICAjIEJ1aWxkIHByaW9ycwogICAgcHJpb3JzIDwtIGJhc2VfcHJpb3JzCiAgICBpZiAoIWlzLm51bGwoYWRkaXRpb25hbF9wcmlvcnMpKSB7CiAgICAgIHByaW9ycyA8LSBjKGJhc2VfcHJpb3JzLCBhZGRpdGlvbmFsX3ByaW9ycykKICAgIH0KICAgIGlmIChvbmx5X3ByaW9ycykgewogICAgICBwcmlvcnMgPC0gTlVMTAogICAgfQogICAgCiAgICAjIEJ1aWxkIGZvcm11bGEKICAgIGFkZGl0aW9uYWxfdmFyaWFibGVzLmZvcm11bGEgPC0gcGFzdGUoYWRkaXRpb25hbF92YXJpYWJsZXMuc29ydGVkLCBjb2xsYXBzZSA9ICIgKyAiKQogICAgZm9ybXVsYSA8LSBiYXNlX2Zvcm11bGEKICAgIGlmICghaXMubnVsbChhZGRpdGlvbmFsX3ZhcmlhYmxlcykpIHsKICAgICAgZm9ybXVsYSA8LSBwYXN0ZShiYXNlX2Zvcm11bGEsIGFkZGl0aW9uYWxfdmFyaWFibGVzLmZvcm11bGEsIHNlcCA9ICIgKyAiKQogICAgfQogICAgCiAgICAjIEJ1aWxkIGNhY2hlIGZpbGUgbmFtZQogICAgYWRkaXRpb25hbF92YXJpYWJsZXMubmFtZSA8LSBwYXN0ZShhZGRpdGlvbmFsX3ZhcmlhYmxlcy5zb3J0ZWQsIGNvbGxhcHNlID0gIi4iKQogICAgbmFtZSA8LSBiYXNlX25hbWUKICAgIGlmICghaXMubnVsbChhZGRpdGlvbmFsX3ZhcmlhYmxlcykpIHsKICAgICAgbmFtZSA8LSBwYXN0ZShiYXNlX25hbWUsIGhhc2goYWRkaXRpb25hbF92YXJpYWJsZXMubmFtZSksIHNlcCA9ICIuIikKICAgIH0KICAgIG5hbWUgPC0gcGFzdGUobmFtZSwgcGFzdGUoInNhbXBsZV9wcmlvcnMtIiwgc2FtcGxlX3ByaW9yLCBzZXAgPSAiIiksIHNlcCA9ICIuIikKICAgIG5hbWUgPC0gcGFzdGUobmFtZSwgcGFzdGUoInByaW9yc19oYXNoLSIsIGhhc2gocHJpb3JzKSwgc2VwID0gIiIpLCBzZXAgPSAiLiIpCiAgICBuYW1lIDwtIHBhc3RlKG5hbWUsIHBhc3RlKCJmb3JtdWxhX2hhc2gtIiwgaGFzaChmb3JtdWxhKSwgc2VwID0gIiIpLCBzZXAgPSAiLiIpCiAgICAKICAgICMgR2V0IGNvbnRyb2wgb3B0aW9ucwogICAgY29udHJvbCA8LSBiYXNlX2NvbnRyb2wKICAgIGlmICghaXMubnVsbChjb250cm9sX292ZXJyaWRlKSkgewogICAgICBjb250cm9sIDwtIGNvbnRyb2xfb3ZlcnJpZGUKICAgIH0KICAgIAogICAgIyBDcmVhdGUgYW5kIHJldHVybiB0aGUgYnJtcyBtb2RlbAogICAgYnJtKAogICAgICBmb3JtdWxhID0gYXMuZm9ybXVsYShmb3JtdWxhKSwKICAgICAgZmFtaWx5ID0gZmFtaWx5LAogICAgICBkYXRhID0gYXMuZGF0YS5mcmFtZShkYXRhKSwKICAgICAgcHJpb3IgPSBwcmlvcnMsCiAgICAgIGVtcHR5ID0gb25seV9wcmlvcnMsCiAgICAgIHNhbXBsZV9wcmlvciA9IHNhbXBsZV9wcmlvciwKICAgICAgZmlsZSA9IHBhc3RlKCJmaXRzIiwgbmFtZSwgc2VwID0gIi8iKSwKICAgICAgZmlsZV9yZWZpdCA9ICJvbl9jaGFuZ2UiLAogICAgICBzZWVkID0gMjAyMTA0MjEsCiAgICAgIGNvbnRyb2wgPSBjb250cm9sCiAgICApCiAgfQp9CmBgYAoKRXhhbXBsZSB1c2FnZToKCmBgYHtyLCBldmFsPUZBTFNFLCBjbGFzcy5zb3VyY2UgPSAnZm9sZC1zaG93J30KIyMgTm90IHJ1bgptLndpdGggPC0gZXh0ZW5kYWJsZV9tb2RlbCgKICBiYXNlX25hbWUgPSAibSIsIAogIGJhc2VfZm9ybXVsYSA9ICJ0aW1lIH4gMSIsCiAgZmFtaWx5ID0gbmVnYmlub21pYWwoKSwgCiAgZGF0YSA9IGQuYm90aF9jb21wbGV0ZWQsIAogIGJhc2VfcHJpb3JzID0gYygKICAgIHByaW9yKG5vcm1hbCgwLCAxKSwgY2xhc3MgPSAiSW50ZXJjZXB0IikKICApCikKCnByaW9yX3N1bW1hcnkobS53aXRoKG9ubHlfcHJpb3JzID0gVFJVRSkpCgpwcF9jaGVjayhtLndpdGgoc2FtcGxlX3ByaW9yID0gIm9ubHkiKSwgbnNhbXBsZXMgPSAyMDApCgpzdW1tYXJ5KG0ud2l0aCgpKQoKcHBfY2hlY2sobS53aXRoKCksIG5zYW1wbGVzID0gMjAwKQoKbG9vKAogIG0ud2l0aCgpLAogIG0ud2l0aCgiaGlnaF9kZWJ0X3ZlcnNpb24iKSwKICBtLndpdGgoYygiaGlnaF9kZWJ0X3ZlcnNpb24iLCAic2NlbmFyaW8iKSkKKQojIyBFbmQoTm90IHJ1bikKYGBgCg==